#!/bin/bash
# Using '/bin/bash' instead of '/usr/bin/env bash' means process gets listed in ps and pstree as cache_dirs instead of bash cache_dirs

#Programmer notes
# Check with https://www.shellcheck.net/

####################################################################################
# cache_dirs
# A utility to attempt to keep directory entries in the linux
# buffer cache to allow disks to spin down and no need to spin-up
# simply to get a directory listing on an unRAID server.
#
# Version 1.0    Initial proof of concept using "ls -R"
# Version 1.1    Working version, using "ls -R" or "find -maxdepth"
# Version 1.2    Able to be used with or without presence of user-shares.
#                Removed "ls -R" as it was too easy to run out of ram. (ask me how I know)
#                Added -i include_dir to explicitly state cached directories
#                Added -v option, verbose statistics when run in foreground
#                Added -q option, to easily terminate a process run in the background
#                Added logging of command line parameters to syslog
# Version 1.3    Added -w option, to wait till array comes online before starting scan
#                of /mnt/disk* share folders.
#                Changed min-seconds delay between scans to 1 instead of 0.
#                Moved test of include/exclude directories to after array is on-line
#                Added logging of mis-spelled/missing include/exclude dirs to syslog
#                Added ability to have shell wildcard expansion in include/exclude names
# Version 1.4    Fix bug with argument order passed to find when using -d option
#                Fixed command submitted to "at" to use full path. Should not need to
#                set PATH variable in "go" script.
#                Added ability to also cache scan /mnt/user with -u option
# Version 1.4.1  Fixed version comment so it is actually a comment.
# Version 1.5    Added -V to print version number.
#                Added explicit cache of root directories on disks and cache drive
#                Modified "average" scan time statistic to be weighted average with a window
#                of recent samples.
#                Added -a args option to allow entry of args to commands after dir/file name
#                example: cache_dirs -a "-ls" -d 3
#                This will execute "find disk/share -ls -maxdepth 3"
# Version 1.6    Fixed bug... if -q was used, and cache_dirs not currently running,
#                it started running in error. OOps... Added the missing "exit"
#                Changed vfs_cache_pressure setting to be 1 instead of 0 by default.
#                Added "-p cache_pressure" to allow experimentation with vfs_cache_pressure values
#                (If not specified, default value of 1 will be used)
#                Made -noleaf the default behavior for the "find" command (use -a "" to disable).
#                Added logic to force all disks "busy" by starting a process with each as their
#                current working directory.   This will prevent a user from seeing a frightening
#                Unformatted description if they attempt to stop the array.  A second "Stop" will
#                succeed (the scan is paused for 2 minutes, so it may be stopped cleanly)
#                Added new -B option to revert to the old behaviour and not force disks busy if by
#                chance this new feature causes problems for some users.
#                Allow min seconds to be equal to max seconds in loop delay range.
#                Added run-time-logging, log name = /var/log/cache_dirs.log
# Version 1.6.1  Fixed bug. Added missing /mnt/cache disk to scanned directories
# Version 1.6.2  Added trap to clean up processes after kill signal when run in background
# Version 1.6.3  Modified to deal with new un-mounting message in syslog in 4.5b7 to
#                allow array shutdown to occur cleanly.
# Version 1.6.4  Modified to suspend scan during time "mover" script is running to prevent
#                DuplicateFile messages from occurring as file is being copied.
#                Added -S option to NOT suspend scan during mover process.
#                Added logic to re-invoke cache_dirs if array is stopped and then re-started
#                by submitting command string to "at" to re-invoke in a minute.
#                Added entry to "usage()" function for -B
# Version 1.6.5  Fixed what I broke in looking for "mover" pid to suspend during the "mover"
#                to eliminate warnings in syslog about duplicate files detected while files were
#                being copied.
# Version 1.6.6  Fixed mover-detection to use the exact same logic as "mover" (and fixed stupid typo I had made)
# Version 1.6.7  Added cache_pressure to "usage" statement, fixed bug where it reverted to 10 after being invoked through "at"
#                when used with the -w option.
# Version 1.6.8  Added -U NNNNN option to set ulimit, and detection of 64 bit OS so odds are this new option will not be needed.
#                by default, ulimit is set to 5000 on 32 bit OS, and 30000 on 64 bit OS.  Either can be over-ridden with -U NNNNN on command line
# Version 1.6.9  Removed exec of /bin/bash.  Newer bash was not setting SHELL variable causng infinate loop if invoked from "go" script.
#                Changed default ulimit on 64 bit systems to 50000.
#                by default, ulimit is now set to 5000 on 32 bit OS, and 50000 on 64 bit OS.  Either can be over-ridden with -U NNNNN on command line
#                Setting ulimit to zero ( with "-U 0" option) is now special, cache_dirs will not set any ulimit at all.  You'll inherit the system value, whatever it might be.
# Joe L.
#
# Version 2.0.0  Added gradual depth to avoid continous scans of filesystem, monitor of disk-idle, and better user-feedback as to disk spin-up in log-file.
#                Now stops cache_dirs immediately on stop signal (eg array stop) including stopping the currently running find-process.
#                Force-disk-busy now defaults no and inverted flag (and changed -B to -b) because it was (mostly) unRaid 4 and its unnessary when using plg with unmount disk event.
# Version 2.0.1  Fixed missing sleep. Now decreases scan-depth after few seconds (20-40s) if cache is lost after many successful cache-hits, because we don't want cache_dirs to be a resource-hog when system is otherwise occupied.
# Version 2.0.2  Fixed looping bash check in unRaid 6, plus fixed some too aggressive depth checks.
# Version 2.0.3  Bugfix suspend mover, and added concise log of lost cache.
# Version 2.0.4  Added more lost cache log, enabled by creating log file /var/log/cache_dirs_lost_cache.log
# Version 2.0.5  Updated for unRaid 6.1.2 new location of mdcmd (used to find idle-times of disks)
# Version 2.0.6  Included original 'B' option, it is unused but kept for compatibility reasons
# arberg
#
# Version 2.1.0  Modified for unRaid V6.1 and above only.
#                Removed V5 specific code.
#                Removed disks busy code.
#                Removed wait for array to come on line.
#                Remove unused variables.
#                Modifications to improve readability.
# Version 2.1.1  Removed additional unused variables.
#                Removed V5 ulimit usage info that doesn't apply.
#                Show type of scanning being done - adaptive or fixed when cache_dirs is started.
# dlandon
#
# Version 2.2.0  When cache lost, don't wait for disks idle, instead decrease depth
#                Now scans each disks and cache in separate processes. Thus removed option -u for scan user
#                Fixed loss of adaptive after 1 week
#                Weekly rescan only when disks been idle long
#                Apaptive scan will not retry increasing level to previous seen stable level until disks have been idle for a long time (20s)
#                -q now also kills subproccesses immediately (so cache_dirs does not prevent stopping the array)
#                Added file-count and depth adjustment by filecount
# Version 2.2.1  Added disabling of multithreaded scan of disks flag -T
#                Added diagnostics generate zip, flag -L
#                Added -P print file count flag
#                Disk idle times sometimes reported as crazy high numbers (when disks spun down on some systems). Report 9999 instead of crazy idle-times
#                Added '-u' option to scan user share again, because some users reports disks spinning up on access of /mnt/user, though not /mnt/disk*
# Version 2.2.2  Added minimum depth param -c for adaptive depth scanning
# Version 2.2.3  Logging
# Version 2.2.4  Added rolled logs to -L diagnostics archive
# Version 2.2.5  Improved diagnostics info. Fixed sleep duration, before fixed depth always slept 1s, now it depends on disks idle and avg scan time.
# Version 2.2.6  CSV logging with all scan timings
# Version 2.2.7  Fixed -a option which was broken in 2.2
# arberg
#
# Version 2.2.8  Added multi-cache pool support
#                change disk* to disk[0-9]* to avoid issues with the UD plugin
# bergware
####################################################################################
version=2.2.8
program_name=$(basename "$0")
arg_count=$#

run_log="/var/log/cache_dirs.log"
csv_log="/var/log/cache_dirs.csv"
lost_cache_log="/var/log/cache_dirs_lost_cache.csv"

usage() {
  echo
  echo "Usage: $program_name [-m min_seconds] [-M max_seconds] [-F] [-d maxdepth(adaptive)] [-D maxdepth(fixed)] [-c command] [-a args] [-e exclude_dir] [-i include_dir]"
  echo "       $program_name -V      = print program version"
  echo "       $program_name -q"
  echo "       $program_name -l on   = turn on logging to $run_log and $lost_cache_log and $csv_log"
  echo "       $program_name -l off  = turn off logging to $run_log and $lost_cache_log and $csv_log"
  echo " -m NN    =   minimum seconds to wait between directory scans (default=1)"
  echo " -M NN    =   maximum seconds to wait between directory scans (default=10)"
  echo " -U NN    =   set ulimit to NN to limit memory used by script (default=50000), '-U 0' sets no ulimit at all)"
  echo " -F       =   do NOT run in background, run in Foreground and print statistics as it loops and scans"
  echo " -v       =   when used with -F, verbose statistics are printed as directories are scanned"
  echo " -s       =   shorter-log - print count of directories scanned to syslog instead of their names"
  echo " -d NN    =   max depth to allow when searching adaptively for appropriate depth level, used in \"find -maxdepth NN\" "
  echo " -D NN    =   sets fixed depth level and disables adaptive depth level, uses \"find -maxdepth NN\" "
  echo " -t NN    =   time in seconds between scheduled scan towards max depth, default weekly; this setting is only relevant with adaptive scan enabled (without -D setting)"
  echo " -W NN    =   Disk Idle Timer - When the disks have been idle this long, cache_dirs adaptive scan starts working towards building the cache."
  echo " -X NN    =   timeout in seconds for direcotry scan during disk-idle periods (and when fixed depth -D is set)"
  echo " -Y NN    =   timeout in seconds for direcotry scan during initial scan and when cache has been lost"
  echo " -Z NN    =   timeout in seconds for direcotry scan after all directories have been cached"
  echo " -c command = use command instead of \"find\" "
  echo "              (\"command\" should be quoted if it has embedded spaces)"
  echo " -a args  =   append args to command"
  echo " -u       =   also scan /mnt/user (scan user shares) - may considerably increase cpu time, and should not be necessary but seem to be for some"
  echo " -e exclude_dir  (may be repeated as many times as desired)"
  echo " -i include_dir  (may be repeated as many times as desired)"
  echo " -p NN    =   set cache_pressure to NN (by default = 10). 0 means never reclaim cache. See 'vfs_cache_pressure' in https://www.kernel.org/doc/Documentation/sysctl/vm.txt"
  echo " -S       =   do not suspend scan during 'mover' process"
  echo ""
  echo "Diagnostics, log:"
  echo " -z       =   concise log (log run criteria on one line)"
  echo " -P       =   compute files in each share and exit. Can be used with maxdepth -d param, and include -i and exclude -e."
  echo " -L       =   generate diagnostics zip-file and exit"
  echo " -T       =   disable multithreaded scan of disks"
  echo " -q       =   terminate any background instance of cache_dirs"
  echo ""
}

background=yes
verbose=no
# min scan seconds ignored, always sleep max, because it always tended towards min_seconds anyway and seems unnecessary
min_seconds=1
max_seconds=10
short_log=no
maxDepthUnbounded=9999
maxDepth="$maxDepthUnbounded"
fixdepth=-1
command="find"
window_array_length=20
avg_elapsed_time=0
exclude_array_count=0
include_array_count=0
quit_flag="no"
suspend_during_mover="yes"
commandargs=$*
args="-noleaf"
concise_log="no"
include_scan_of_user_share=0
# When min_disk_idle_before_restarting_scan_sec was 4 it restarting scanning too soon
min_disk_idle_before_restarting_scan_sec=60
# scan_timeout_sec_stable used for busy rescan and stable scan, scan_timeout_sec_idle used for initial on disks idle scan
scan_timeout_sec_busy=30
scan_timeout_sec_idle=150
scan_timeout_sec_stable=30
slow_scan_time_limit=15
# Adaptively increase depth by 1 until depth_max_incremental_depth, then go to $maxDepth
depth_max_incremental_depth=20
frequency_of_full_depth_scan_sec=$((7*24*3600))
MODE_IDLE_RESCANNING=1
MODE_BUSY_RESCANNING=2
MODE_STABLE=3
MODE_STABLE_IDLE=4
mode=$MODE_BUSY_RESCANNING
no_idle_scans_increase_depth=3
no_busy_scans_increase_depth=50
# Note that we decrease depth immediately if a scan has a timeout. A slow scan means its lasts longer than slow_scan_time_limit=15
no_slow_scans_decrease_depth_idle=4
no_slow_scans_decrease_depth_busy=4
no_slow_scans_decrease_depth_stable=4
min_no_scans_at_new_level=3
multithreaded_scan=1
ulimit_mem=50000
do_print_file_counts=0
minDepth="4"

# Constants
NANO_PR_SEC=1000000000
NANO_PR_MS=1000000

# cache_pressure of 0 will potentially run out of RAM if a large directory is scanned and not enough RAM
# esists. User processes will then be killed to free space rather than cache freed.
# (It has happened several times on my server when I forgot to exclude my data folder.
# It is not fun trying to regain control without a full reboot.  I've changed the default to "1" instead. )
# If you have enough RAM, and few enough files being cached, you can specify "-p 0" on the command line
# to set the vfs_cache_pressure to 0.  Otherwise, this default value of 1 should prevent memory starvation
# and the OOM (out-of-memory) state killing on your processes to free up some RAM.
# 1 did not do it with my 500Meg of RAM... trying cache_pressure of 10, use -p 1 if you still want the old value
cache_pressure=10

# Array & Pool devices
array=/mnt/disk[0-9]*
pools=$(ls -d /mnt/*|grep -Pv '/(disk[0-9]+|disks|user0?)$')

verbose_echo() {
  [ $background = "no" ] && [ $verbose = "yes" ] && echo "$1"
}

log() {
  [ $background = "no" ] && [ $verbose = "yes" ] && echo "$*"
  [ "$run_log" != "" ] && [ -f "$run_log" ] && echo "$*" >> $run_log
}


syslog() {
  log $1
  echo "$1" | logger "-t$program_name"
}

syslogerror() {
  [ "$run_log" != "" ] && [ -f "$run_log" ] && echo "$*" >> $run_log
  echo "$1" | logger "-t$program_name"
}

logCsvFile() {
  theLogFile="$1"
  shift
  if [ "$theLogFile" != "" ] && [ -f "$theLogFile" ] ; then
    first=1
    for var in "$@"
    do
      if ((first == 0)) ; then
        echo -n ';'
      fi
      echo -n "\"$var\""
      first=0
    done >> $theLogFile
    echo >> $theLogFile
  fi
}

logLostCache() {
  logCsvFile $lost_cache_log $*
}


logLostCacheSimple() {
  logLostCache $start_time_txt ${elapsed_secs} ${prev_sleep_duration} ${time_since_disk_access_before_scan_sec} ${time_since_disk_access_after_scan_sec} $appliedDepth $appliedMaxDepthCurrent $appliedMaxDepth $1 $depth_slow_scan_counter $depth_success_idle_incr_counter
}

logLostCacheHeader() {
  # Cannot be logged with logCsv because the for-loop breaks on spaces, not param numbers
  if [ "$lost_cache_log" != "" ] && [ -f "$lost_cache_log" ] ; then
    echo '"Date";"Time";"ScanTime";"PrevSleep";"IdleTimeBeforeScan";"IdleTimeAfterScan";"Depth";"MaxDepthCurrent";"MaxDepthWeek";"ForcedRestartDepthScan";"Scans at depth";"Scans succes count"' >> $lost_cache_log
  fi
}

logCsv() {
  logCsvFile "$csv_log" $*
}
logCsvSimple() {
  logCsv $(date "+%Y.%m.%d") $(date "+%H:%M:%S") ${elapsed_secs} ${prev_sleep_duration} ${time_since_disk_access_before_scan_sec} ${time_since_disk_access_after_scan_sec} $appliedDepth $appliedMaxDepthCurrent $appliedMaxDepth $depth_slow_scan_counter $depth_success_idle_incr_counter
}

logCsvHeader() {
  # Cannot be logged with logCsv because the for-loop breaks on spaces, not param numbers
  if [ "$lost_cache_log" != "" ] && [ -f "$lost_cache_log" ] ; then
    echo '"Date";"Time";"ScanTime";"PrevSleep";"IdleTimeBeforeScan";"IdleTimeAfterScan";"Depth";"MaxDepthCurrent";"MaxDepthWeek";"Scans at depth";"Scans at depth with idle disks"' >> $csv_log
  fi
}

run_diagnostics() {
  echo "################## PS #######################"
  psCommand="ps -e x -o ppid,pid,pgid,%cpu,%mem,tty,vsz,rss,etime,cputime,rgroup,ni,fname,args"
  $psCommand | head -1
  $psCommand | grep "cache_dirs\|find\|wc"
  echo "################## PSTREE #######################"
  pstree -p | grep -2 cache_dirs
  echo "################## TOP #######################"
  top -b -n1 | head -20
  echo "################## MEMORY #######################"
  free -m
  echo "################## unRAID Version #######################"
  cat /proc/version
  head -1 /boot/changes.txt
}

while getopts "a:p:m:M:c:d:D:e:i:l:FszBbwuvVhqSc:U:t:W:X:Y:Z:LTP" opt; do
  case $opt in
  c ) minDepth="$OPTARG"
    ((minDepth < 1)) && minDepth=1
    ;;
  m ) min_seconds=$OPTARG ;;
  M ) max_seconds=$OPTARG ;;
  F ) background=no ;;
  v ) verbose=yes ;;
  V ) echo "$program_name version: $version"; exit 0 ;;
  u ) include_scan_of_user_share=1 ;;
  c ) command="$OPTARG" ;;
  a ) args="$OPTARG" ;;
  d ) maxDepth=$OPTARG
    (( maxDepth <= depth_max_incremental_depth )) && depth_max_incremental_depth=$((maxDepth - 1))
    (( depth_max_incremental_depth == 0 )) && depth_max_incremental_depth=1
    command="find" ;;
  D ) fixdepth=$OPTARG
    command="find" ;;
  i ) include_array[$include_array_count]="$OPTARG"
    include_array_count=$((include_array_count+1)) ;;
  e ) exclude_array[$exclude_array_count]="$OPTARG"
    exclude_array_count=$((exclude_array_count+1)) ;;
  h ) usage >&2 ; exit 0 ;;
  p ) cache_pressure="$OPTARG" ;;
  U ) ulimit_mem="$OPTARG" ;;
  q ) quit_flag="yes" ;;
  w ) ;; # unused, kept for compatibility reasons
  s ) short_log="yes" ;;
  B ) ;; # unused, kept for compatibility reasons
  b ) ;; # unused, kept for compatibility reasons
  S ) suspend_during_mover="no" ;;
  z ) concise_log="yes" ;;
  t ) frequency_of_full_depth_scan_sec="$OPTARG" ;;
  l )
    case "$OPTARG" in
    on)
      touch $run_log
      touch $lost_cache_log
      touch $csv_log
      echo "Logging enabled to $run_log and $lost_cache_log and $csv_log"
      ;;
    off)
      rm "$run_log"
      rm "$lost_cache_log"
      rm "$csv_log"
      echo "Logging to $run_log stopped"
      ;;
    *)
      echo "Invalid argument to -l option"
      echo "Usage:" >&2
      echo "cache_dirs -l on" >&2
      echo "or" >&2
      echo "cache_dirs -l off" >&2
      exit 2
      ;;
    esac
    ;;
  L )
    run_diagnostics | tee cache_dirs_diagnostics_processes.log
    [ -f cache_dirs_diagnostics.zip ] && rm cache_dirs_diagnostics.zip
    zip -j cache_dirs_diagnostics.zip cache_dirs_diagnostics_processes.log /var/log/syslog /var/log/cache_dirs*
    rm cache_dirs_diagnostics_processes.log
    echo
    echo "Generated cache_dirs_diagnostics.zip"
    exit 0
    ;;
  P ) do_print_file_counts=1 ;;
  T ) multithreaded_scan=0 ;;
  W ) min_disk_idle_before_restarting_scan_sec="$OPTARG" ;;
  X ) scan_timeout_sec_idle="$OPTARG" ;;
  Y ) scan_timeout_sec_busy="$OPTARG" ;;
  Z ) scan_timeout_sec_stable="$OPTARG" ;;
  \?) usage >&2 ; exit ;;
  esac
done

if [ "$(whoami)" != "root" ]
then
  echo "ERROR: Run as root"
  exit 1
fi

function join { local IFS="$1"; shift; echo "$*"; }

build_dir_list() {
  # build a list of directories to cache.
  #   If no "-i" options are given, this will be all the top level directories in $array and $pools
  #   If "-i" entries are given, they will be the only top level dirs cached.
  #   If "-e" (exclude) directories are given, they are then deleted from the list by the comm -23 coommand.
  if [ $include_array_count -gt 0 ] ; then
    top_dirs=`(
    # Include designated directories
    a=0
    while test $a -lt $include_array_count
    do
      included_excl=$(find $array $pools -type d -maxdepth 1 -mindepth 1 -name "${include_array[$a]}" -exec basename {} \; 2>/dev/null)
      echo "$included_excl" | sort -u
      a=$((a+1))
    done
    )| sort -u`
  else
    top_dirs=$(find $array $pools -type d -maxdepth 1 -mindepth 1  -exec basename {} \; 2>/dev/null | sort -u)
  fi

  exclude_dirs=`(
    # Exclude designated directories from being processed
    a=0
    while test $a -lt $exclude_array_count
    do
      expanded_excl=$(find $array $pools -type d -maxdepth 1 -mindepth 1 -name "${exclude_array[$a]}" -exec basename {} \; 2>/dev/null)
      echo "$expanded_excl" | sort -u
      a=$((a+1))
    done
  )| sort -u`
  scan_dirs=$(comm -23 <(echo "$top_dirs") <(echo "$exclude_dirs"))
  echo "$scan_dirs"
}

if (( do_print_file_counts == 1 )) ; then
  dir_list=$(build_dir_list)
  depth_arg=""
  if [ "$maxDepth" -ne "$maxDepthUnbounded" ] ; then
    depth_arg="-maxdepth $maxDepth"
    echo "Computing number of files until max depth $maxDepth"
  else
    echo "Computing number of files"
  fi
  for share_dir in $dir_list
  do
    echo $share_dir: $(find $array/$share_dir $(echo $pools|tr ' ' '\n'|awk '{print $1"/$share_dir"}') $depth_arg 2>/dev/null | wc -l )
  done
  exit 0
fi

# We need to use it with -KILL
killtree() {
  local _pid=$1
  local _sig=${2:--TERM}
  kill -stop ${_pid} # needed to stop quickly forking parent from producing children between child killing and parent killing
  for _child in $(ps -o pid --no-headers --ppid ${_pid}); do
    killtree ${_child} ${_sig}
  done
  kill ${_sig} ${_pid}
}

lockfile="/var/lock/cache_dirs.LCK"
if [ -f "${lockfile}" ] ; then
  # The file exists so read the PID
  # to see if it is still running
  lock_pid=$(head -n 1 "${lockfile}")

  pid_running=$(ps -p "${lock_pid}" | grep ${lock_pid})

  if [ -z "${pid_running}" ] ; then
    if [ "$quit_flag" = "no" ] ; then
      # The process is not running
      # Echo current PID into lock file
      echo $$ > "${lockfile}"
    else
      echo "$program_name ${lock_pid} is not currently running "
      rm "${lockfile}"
      exit 0
    fi
  else
    if [ "$quit_flag" = "yes" ] ; then
      echo "Stopping $program_name process $lock_pid"
      syslog "Stopping $program_name process $lock_pid"
      # 1. Remove lock-file so we don't spawn new find-processes
      rm "${lockfile}"
      # Sleep 1 because it means we will get log info if slow find processes isn't running
      sleep 1;
      killtree "$lock_pid" -KILL
      exit 0
    else
      echo "$program_name is already running [${lock_pid}]"
      exit 2
    fi
  fi
else
  if [ "$quit_flag" = "yes" ] ; then
    echo "$program_name not currently running "
    exit 0
  else
    echo $$ > "${lockfile}"
  fi
fi

log "$program_name version $version"
#Try to play nice
if [ "$ulimit_mem" -gt 0 ] ; then
  log "Setting Memory ulimit to $ulimit_mem"
  ulimit -v "$ulimit_mem"
else
  log "No Memory ulimit applied"
fi

# validate the cache pressure
cc="$(echo $cache_pressure | sed 's/[0-9]//g')"
if [ ! -z "$cc" ] ; then
  echo "error: cache_pressure must be numeric." >&2
  usage >&2
  exit 2
fi

# validate the min number of seconds
cc="$(echo $min_seconds | sed 's/[0-9]//g')"
if [ ! -z "$cc" ] ; then
  echo "error: min number of seconds must be numeric (whole number, not negative)." >&2
  usage >&2
  exit 2
fi

# validate the max number of seconds
cc="$(echo $max_seconds | sed 's/[0-9]//g')"
if [ ! -z "$cc" ] ; then
  echo "error: max number of seconds must be numeric." >&2
  usage >&2
  exit 2
fi
if [ $max_seconds -lt $min_seconds ] ; then
  echo "error: max number of seconds must be greater than or equal min number of seconds." >&2
  usage >&2
  exit 2
fi

# validate the maxDepth
cc="$(echo $maxDepth | sed 's/[0-9]//g')"
if [ ! -z "$cc" ] ; then
  echo "error: directory scan maxdepth must be numeric." >&2
  usage >&2
  exit 2
fi
cc="$(echo $frequency_of_full_depth_scan_sec | sed 's/[0-9]//g')"
if [ ! -z "$cc" ] ; then
  echo "error: scheduled rescan time must be numeric, with -t $frequency_of_full_depth_scan_sec" >&2
  usage >&2
  exit 2
fi
if (( minDepth > maxDepth )) ; then
  syslog "WARNING: minDepth > maxDepth - corrected"
  minDepth="$maxDepth"
fi

shift $((OPTIND - 1))

# start out in the middle of the range allowed
#num_seconds=$((( $max_seconds + $min_seconds ) / 2 ))
num_seconds=$max_seconds

log "Setting cache_pressure=$cache_pressure"
sysctl vm.vfs_cache_pressure=$cache_pressure >/dev/null 2>&1

# use function like this: result=$(fnc_time_since_last_disk_access)
fnc_time_since_last_disk_access() {
  # check if array is mounted by checking for disk1 directory
  if [ -d /mnt/disk1 ] ; then
    # UnRaid >= 6.1.2 (I think)
    mdcmd_cmd=/usr/local/sbin/mdcmd

    # rdevLastIO will be non-zero if a disk is spinning, it will be the timestamp of last IO (in seconds since epoch)
    last=$($mdcmd_cmd status | grep -a rdevLastIO | grep -v '=0')
    lastAccess="$(echo $last | awk '{t=systime(); gsub("rdevLastIO..=",""); for(i = 1; i <= NF; i++) a[++y]=$i}END{c=asort(a); if (NF > 0) print t-a[NF]; else print 9998; }')"
    if (( lastAccess > 9999)) ; then
      echo "9999"
    else
      echo "$lastAccess"
    fi
    # Code to log all disk ages
    # ages=$(echo $last | awk '{ t=systime(); for(i = 1; i <= NF; i++){ match($i, /rdevLastIO.([0-9]+)/, capgroups); gsub("rdevLastIO..=","", $i);  print capgroups[1] "=" t-$i } }') #print "diskage" i "=" t-$i
    # log "Ages: $ages"
  else
    # Array is not started
    echo "9997"
  fi
}

depthMinusOne() {
  depth=$1
  if (( depth <= minDepth )) ; then
    echo "$minDepth"
  elif (( depth == maxDepth)) ; then
    echo "$depth_max_incremental_depth"
  else
    echo $(( depth - 1 ))
  fi
}

depthMinusPercent() {
  depth="$1"
  percent="$2"
  if (( percent <=0 || percent >= 100 )); then
    log "error percent $percent"
    percent=20
  fi
  local filesAtDepth="${files_at_depth_map[$depth]}"
  local depth_minus_one_value
  depth_minus_one_value=$(depthMinusOne "$depth")
  local result=$minDepth
  for ((i=depth_minus_one_value; i > 1 && i >= minDepth; i--)) ; do
    # bash need $ on $appliedDepth inside hashmap inside (())
    if (( files_at_depth_map[$i] <= filesAtDepth*(100-percent)/100 )); then
      result=$i
      break;
    fi
  done
  echo "$result"
}

depthMinus20Percent() {
  echo "$(depthMinusPercent $1 20)"
}

depthMinus50Percent() {
  echo "$(depthMinusPercent $1 50)"
}


depthPlusOne() {
  depth=$1
  if (( depth == depth_max_incremental_depth || depth == maxDepth )) ; then
    echo "$maxDepth"
  else
    echo $(( depth + 1 ))
  fi
}

function wait_and_get_exit_codes() {
    children=("$@")
    LAST_EXIT_CODE=0
    for job in "${children[@]}"; do
       CODE=0;
       wait ${job} || CODE=$?
       if [[ "${CODE}" != "0" ]]; then
           LAST_EXIT_CODE=1;
       fi
   done
}

do_deep_scan() {
  DEBUG_THREAD=0
  depth_num=$1
  scan_timeout=$2
  depth_arg=""
  [ "$depth_num" -ne "$maxDepthUnbounded" ] && depth_arg="-maxdepth $depth_num"
  scanned_depth_msg+=" depth $depth_num"
  is_last_depth_scan_timed_out=0
  # will update dir_list on each scan, in case new shares have been added
  dir_list=$(build_dir_list)
  scan_start=$(date +%s%N)
  children_pids=()
  for i in $array $pools
  do
    {

      for share_dir in $dir_list
      do
        dir_to_scan="$i/$share_dir"
        # if lockfile removed, then don't do new finds
        [ ! -f "$lockfile" ] && log "breaking scan due to exit request: ${dir_to_scan}" && continue

        # If the directory does not exist on this disk, don't do recursive "directory scan"
        [ ! -d "$dir_to_scan" ] && continue

        current_time_nano=$(date +%s%N)
        # +1 to timeout because this is an integer computation
        remaining_time=$(( ((scan_timeout+1)*NANO_PR_SEC-(current_time_nano-scan_start) ) / NANO_PR_SEC ))
        # Debug logging
        # (( $DEBUG_THREAD )) && log "scanning $depth_num $dir_to_scan - remaining_time=$remaining_time - pid=$BASHPID"
        #verbose_echo "$start_time_txt Executing $command $i/$share_dir $args $depth_arg"
        if (( remaining_time > 0 )) ; then
          # Perform a recursive "find" on /mnt/disk??/share, or /mnt/user/share, or /mnt/cache/share
          if [ -f /bin/timeout ] ; then
            # Stop scan after n seconds. Should actually decrease wait-duration based on previous shares scan-time
            # Note timeout changes the pgid of its process, so cannot kill it using pgid
            /bin/timeout $remaining_time $command "$dir_to_scan" $args $depth_arg >/dev/null 2>&1
          else
            $command "$dir_to_scan" $args $depth_arg >/dev/null 2>&1
          fi
        fi
      done
    } &
    children_pids+=("$!")
    if (( ! multithreaded_scan )) ; then
      wait $!
    fi
  done
  wait_and_get_exit_codes "${children_pids[@]}"
  (( $DEBUG_THREAD )) && log "Finished all depth scan threads: ${children_pids[@]}"

  if (( include_scan_of_user_share == 1 )) ; then
    current_time_nano_disks_done=$(date +%s%N)
    timepassed_disks_total=$(( (current_time_nano_disks_done - scan_start)/ NANO_PR_MS ))

    i="/mnt/user"
    # Note the we scan the user share sequentially to avoid running multiple scans on the same disk at a time where as the disks above are scanned concurrently
    ############################## Copied Above ############################## (ie. don't edit without copying)
    for share_dir in $dir_list
    do
      dir_to_scan="$i/$share_dir"
      # if lockfile removed, then don't do new finds
      [ ! -f "$lockfile" ] && log "breaking scan due to exit request: ${dir_to_scan}" && continue

      # If the directory does not exist on this disk, don't do recursive "directory scan"
      [ ! -d "$dir_to_scan" ] && continue

      current_time_nano=$(date +%s%N)
      # +1 to timeout because this is an integer computation
      remaining_time=$(( ((scan_timeout+1)*NANO_PR_SEC-(current_time_nano-scan_start) ) / NANO_PR_SEC ))
      # Debug logging
      # (( $DEBUG_THREAD )) && log "scanning $depth_num $dir_to_scan - remaining_time=$remaining_time - pid=$BASHPID"
      #verbose_echo "$start_time_txt Executing $command $i/$share_dir $args $depth_arg"
      if (( remaining_time > 0 )) ; then
        # Perform a recursive "find" on /mnt/disk??/share, or /mnt/user/share, or /mnt/cache/share
        if [ -f /bin/timeout ] ; then
          # Stop scan after n seconds. Should actually decrease wait-duration based on previous shares scan-time
          # Note timeout changes the pgid of its process, so cannot kill it using pgid
          /bin/timeout $remaining_time $command "$dir_to_scan" $args $depth_arg >/dev/null 2>&1
        else
          $command "$dir_to_scan" $args $depth_arg >/dev/null 2>&1
        fi
      fi
    done
    ############################## Copied Above END ##############################
    timepassed_user_total=$(( ($(date +%s%N) - current_time_nano_disks_done) / NANO_PR_MS ))
    logline_deep_scan_detailed_info=" (disks $(($timepassed_disks_total/1000))s + user $(($timepassed_user_total/1000))s)"
    # log "total disk+$logline_deep_scan_detailed_info"
  fi

  current_time_nano=$(date +%s%N)
  timepassed_total=$(( (current_time_nano - scan_start) ))
  if [ $LAST_EXIT_CODE -ne 0 ] ; then
    is_last_depth_scan_timed_out=1
    scanned_depth_msg+="(timeout ${scan_timeout}s:Error=$LAST_EXIT_CODE)"
  elif (( timepassed_total >= scan_timeout * NANO_PR_SEC )) ; then
    is_last_depth_scan_timed_out=1
    scanned_depth_msg+="(timeout ${scan_timeout}s)"
  fi
}

# function: exists key in map
exists() {
  if [ "$2" != in ]; then
    echo 'Incorrect usage.'
    echo 'Correct usage: exists {key} in {array}'
    echo 'examlpe1:  if exists $i in files_at_depth_map; then echo "It exists" ; fi'
    echo 'examlpe1a: exists $i in files_at_depth_map && echo "It exists"'
    echo 'examlpe2: if ! exists $i in files_at_depth_map; then echo "Note there" ; fi'
    return
  fi
  eval '[ ${'$3'[$1]+muahaha} ]'
}

count_files() {
  local depth_arg=""
  [ "$1" -ne "$maxDepthUnbounded" ] && depth_arg="-maxdepth $1"
  totalCount=0
  for i in $array $pools; do
    for share_dir in $dir_list ; do
      dir_to_scan="$i/$share_dir"
      if [ -d "$dir_to_scan" ] ; then
        local thisDirCount=$(find $dir_to_scan -type f $depth_arg | wc -l)
        totalCount=$((totalCount+thisDirCount))
      fi
    done
  done
  echo "$totalCount"
}

# does not work as function that echo result, missing cpu_last
# First cpu measurement will be wrong, missing last
update_cpu_usage() {
  cpu_now=($(head -n1 /proc/stat))
  # Get all columns but skip the first (which is the "cpu" string)
  cpu_sum="${cpu_now[@]:1}"
  # Replace the column seperator (space) with +
  cpu_sum=$((${cpu_sum// /+}))
  # Get the delta between two reads
  cpu_delta=$((cpu_sum - cpu_last_sum))
  # Get the idle time Delta
  cpu_idle=$((cpu_now[4]- cpu_last[4]))
  # Calc time spent working
  cpu_used=$((cpu_delta - cpu_idle))
  # Calc percentage
  cpu_usage=$((100 * cpu_used / cpu_delta))

    # Keep this as last for our next read
    cpu_last=("${cpu_now[@]}")
    cpu_last_sum=$cpu_sum
}

format_two_digit() {
  echo "$(echo $1 | awk '{input=$1} END {printf "%2.0f", input}')"
}

# will update dir_list on each scan, in case new shares have been added
dir_list=$(build_dir_list)

if [ "$short_log" = "no" ] ; then
  log_list="$dir_list"
else
  log_list=$(echo "$dir_list" | wc -l)
  log_list=$(echo $log_list " directories cached")
fi

if ((fixdepth == -1)) ; then
  scan_type="adaptive"
  scan_depth=$maxDepth
else
  scan_type="fixed"
  scan_depth=$fixdepth
fi

if (( scan_depth == 9999 )) ; then
  scan_depth="none"
fi

if [ "$concise_log" = "no" ] ; then
  syslog "Arguments=$commandargs"
  syslog "Max Scan Secs=$max_seconds, Min Scan Secs=$min_seconds"
  syslog "Scan Type=$scan_type"
  if ((fixdepth == -1)) ; then
    syslog "Min Scan Depth=$minDepth"
  fi
  syslog "Max Scan Depth=$scan_depth"
  syslog "Use Command='$command $args'"
  syslog "---------- Caching Directories ---------------"
  # syslog "$(join , ${dir_list[@]})"
  syslog "$log_list"
  syslog "----------------------------------------------"
  syslog "Setting Included dirs: $(join , ${include_array[@]})"
  syslog "Setting Excluded dirs: $(join , ${exclude_array[@]})"
  syslog "min_disk_idle_before_restarting_scan_sec=$min_disk_idle_before_restarting_scan_sec"
  syslog "scan_timeout_sec_idle=$scan_timeout_sec_idle"
  syslog "scan_timeout_sec_busy=$scan_timeout_sec_busy"
  syslog "scan_timeout_sec_stable=$scan_timeout_sec_stable"
  syslog "frequency_of_full_depth_scan_sec=$frequency_of_full_depth_scan_sec"
  if (( include_scan_of_user_share == 1 )) ; then
    syslog "Including /mnt/user in scan"
  fi
else
  echo "Arguments=$commandargs, Version=$version, Cache Pressure=$cache_pressure, Max Scan Secs=$max_seconds, Min Scan Secs=$min_seconds, Scan Type=$scan_type, Max Depth=$scan_depth, Use Command='$command $args'" | paste -s -d "," - | logger "-t$program_name"
  echo "$log_list" | paste -s -d "," - | logger "-t$program_name"
fi

logLostCacheHeader
logCsvHeader

a=0
while test $a -lt $exclude_array_count
do
  list=$(eval ls $array/"${exclude_array[$a]}" $(echo $pools|tr ' ' '\n'|awk '{print $1"\"/${exclude_array[$a]}\""}') 2>/dev/null)
  if [ "$list" = "" ] ; then
    syslogerror "ERROR: excluded directory '${exclude_array[$a]}' does not exist."
  fi
  a=$((a+1))
done

a=0
while test $a -lt $include_array_count
do
  list=$(eval ls $array/"${include_array[$a]}" $(echo $pools|tr ' ' '\n'|awk '{print $1"\"/${include_array[$a]}\""}') 2>/dev/null)
  if [ "$list" = "" ] ; then
    syslogerror "ERROR: included directory '${include_array[$a]}' does not exist."
  fi
  a=$((a+1))
done

function get_scan_timeout() {
  if ((fixdepth >= 0)) ; then
    echo "${scan_timeout_sec_idle}"
  elif (( mode == MODE_IDLE_RESCANNING || mode == MODE_STABLE_IDLE )) ; then
    echo "${scan_timeout_sec_idle}"
  elif (( mode == MODE_BUSY_RESCANNING )) ; then
    echo "${scan_timeout_sec_busy}"
  elif (( mode == MODE_STABLE )) ; then
    echo "${scan_timeout_sec_stable}"
  else
    log "Error unknown mode '$mode'"
    echo "${scan_timeout_sec_stable}"
  fi
}

# Internal vars
# isPreviousMaxDepthComputed will be true(1) if we have reached the depth we currently think of as our max depth, ie. appliedDepth == appliedMaxDepth
# - appliedMaxDepth is computed once, and after that we always try to regain that depth, which would cause load on system, unless we wait for idle disks I suppose
depth_scan_counter=0
depth_success_idle_incr_counter=0
depth_success_busy_incr_counter=0
depth_slow_scan_counter=0
if ((fixdepth >= 0)) ; then
  appliedDepth=$fixdepth
  maxDepth=$fixdepth
  isPreviousMaxDepthComputed=1
  scan_timeout_sec=${scan_timeout_sec_idle}
else
  appliedDepth=$minDepth
  scan_timeout_sec=${scan_timeout_sec_busy}
  isPreviousMaxDepthComputed=0
fi

# invariant appliedMaxDepthCurrent < appliedMaxDepth <= maxDepth
# appliedMaxDepthCurrent: Current stable set, will be increased after long disk idle
# appliedMaxDepth: Best seen appliedMaxDepth, reset once a week
# maxDepth: User configured max depth
appliedMaxDepth=$maxDepth
appliedMaxDepthCurrent=$appliedMaxDepth
last_scan_towards_max_depth=$(date +%s)
is_last_depth_scan_timed_out=0
prev_sleep_duration=0
next_sleep_duration=0

declare -A files_at_depth_map

{
  log "cache_dirs started"
  while [ -f "$lockfile" ]
  do
    if [ "$suspend_during_mover" = "yes" ] ; then
      if [ -f /var/run/mover.pid ] ; then
        if ps h $(cat /var/run/mover.pid) | grep mover >/dev/null 2>&1 ; then
          log "Suspended during moving, now sleeping 10 seconds"
          sleep 10
          continue
        fi
      fi
    fi

    # always cache root dirs on each of the disks
    for i in $array $pools
    do
      find $i -maxdepth 1 -noleaf >/dev/null 2>/dev/null
    done

    time_since_disk_access_before_scan_sec=$(fnc_time_since_last_disk_access)

    ############## Here the actual find is executed ################
    start_time_nano=$(date +%s%N)
    start_time_txt=$(date "+%Y.%m.%d %H:%M:%S")

    scan_timeout_sec=$(get_scan_timeout)
    scanned_depth_msg=""
    # I tried rescan at lower depth if scan timed out, but it hurt cache, some depth levels were unattainable with the lower depth scan enabled, they must have evicted higher depth cache for some reason
    do_deep_scan $appliedDepth $scan_timeout_sec
    # Count number of files at depth, the first time depth is used
    if ! exists "$appliedDepth" in files_at_depth_map ; then
      files_at_depth_map[$appliedDepth]+="$(count_files $appliedDepth)"
      if ((appliedDepth == maxDepth)) ; then
        depth_max_incremental_depth=$(depthMinus20Percent $maxDepth)
        #################################
        # Debugging of functions below
        # log "depth_max_incremental_depth=$depth_max_incremental_depth"

        # # Debug logging of levels
        # for ((i=1; i < depth_max_incremental_depth; i++)) ; do
        #   # bash need $ on $appliedDepth inside hashmap inside (())
        #   log "depthMinusOne($i)=$(depthMinusOne $i)"
        # done
        # log "depthMinusOne($maxDepth)=$(depthMinusOne $maxDepth)"

        # for ((i=1; i < depth_max_incremental_depth; i++)) ; do
        #   # bash need $ on $appliedDepth inside hashmap inside (())
        #   log "depthMinus20Percent($i)=$(depthMinus20Percent $i)"
        # done
        # log "depthMinus20Percent($maxDepth)=$(depthMinus20Percent $maxDepth)"

        # for ((i=1; i < depth_max_incremental_depth; i++)) ; do
        #   # bash need $ on $appliedDepth inside hashmap inside (())
        #   log "depthMinus50Percent($i)=$(depthMinus50Percent $i)"
        # done
        # log "depthMinus50Percent($maxDepth)=$(depthMinus50Percent $maxDepth)"
        #################################
      fi
    fi

    time_since_disk_access_after_scan_sec=$(fnc_time_since_last_disk_access)
    end_time_nano=$(date +%s%N)

    # track how long the recursive "directory scan" is taking.  If it starts to take longer it must be
    # because it has to read more from the physical disk.  If so, adjust the timing to
    # perform the directory scan more frequently.
    elapsed_time=$(( end_time_nano-start_time_nano ))
    elapsed_secs=$(( elapsed_time / NANO_PR_SEC ))
    were_disks_idle_during_scan=$((time_since_disk_access_after_scan_sec>=elapsed_secs))

    # Only update avg scan time when disks were idle during scan. This gives us ability to judge whether disks were accessed due to cache_dirs scan even if other processes access disks
    if (( were_disks_idle_during_scan )) ; then
      alen=${#avg[@]}
      # Move all the counts up one position in the array.
      for (( i = alen ; i > 0 ; i-- ))
      do
        [ $i -lt $window_array_length ] && avg[$((i+1))]=${avg[$i]}
      done

      # The newest will always be stored at index 1
      avg[1]=$elapsed_time

      # get the weighted average of the last $window_array_length loop passes
      # more recent values count far more than older values.
      tot_time=0
      alen=${#avg[@]}

      tot_count=0
      for (( i = 1; i <= alen; i++ ))
      do
        weight=$(( alen - i + 1 ))
        weight=$(( weight * 3 ))
        tot_count=$(( tot_count + weight))
        tot_time=$(( tot_time + avg[i] * weight ))
      done
      avg_elapsed_time=$((tot_time/tot_count))
    elif [ $avg_elapsed_time -eq 0 ] ; then
      avg_elapsed_time=$elapsed_time
    fi
    avg_elapsed_secs=$(( avg_elapsed_time / NANO_PR_SEC ))

    # Only decrease sleep when non-idle scan, note avg is computed over scan with idle disks, but not necessary at current depth level
    if (( !were_disks_idle_during_scan && avg_elapsed_time * 3 < elapsed_time && num_seconds > min_seconds )) ; then
      num_seconds=$min_seconds
    fi
    if (( avg_elapsed_time + NANO_PR_SEC > elapsed_time && num_seconds < max_seconds )) ; then
      num_seconds=$max_seconds
    fi

    is_slow_scan=0
    is_depth_reduced=0
    log_disk_access_msg=""
    ######### Update mode and hasBeenIdleLong ###
    if ((fixdepth == -1)) ; then
      (( time_since_disk_access_before_scan_sec >= min_disk_idle_before_restarting_scan_sec )) && hasBeenIdleLong=1 || hasBeenIdleLong=0
      if (( depth_scan_counter >= min_no_scans_at_new_level )) ; then
        # never change mode before having tried a few times to build cache pressure at level
        if (( hasBeenIdleLong )) ; then
          if (( mode == MODE_STABLE )) ; then
            mode=$MODE_STABLE_IDLE
          elif (( mode == MODE_BUSY_RESCANNING )) ; then
            mode=$MODE_IDLE_RESCANNING
          fi
        elif (( time_since_disk_access_before_scan_sec == 0 )) ; then
          if ((mode == MODE_IDLE_RESCANNING)) ; then
            # We have slept a second, so if disks have not been idle for a second, it means they are busy again. It does not hurt if we move back and forth between the two modes
            if (( appliedDepth <= 3 )) ; then
              mode=$MODE_BUSY_RESCANNING
            else
              # It seems to work great to do all scanning work when we know system is idle, and stop immediately with the scanning of deeper levels when disks are not idle.
              mode=$MODE_STABLE
            fi
          elif ((mode == MODE_STABLE_IDLE)) ; then
            # It seems to work great to do all scanning work when we know system is idle, and stop immediately with the scanning of deeper levels when disks are not idle.
            mode=$MODE_STABLE
          fi
        fi
      fi
    else
      if (( time_since_disk_access_after_scan_sec >= 2 || avg_elapsed_secs + 2 >= elapsed_secs )) ; then
        mode=$MODE_STABLE_IDLE
      else
        mode=$MODE_BUSY_RESCANNING
      fi
      # log "Fixed sleep time_since_disk_access_after_scan_sec($time_since_disk_access_after_scan_sec) >= 2 || avg_elapsed_secs($avg_elapsed_secs) + 2 >= elapsed_secs($elapsed_secs) => mode=$mode"
    fi

    current_time_sec=$(date +%s)
    skip_sleep=0
    if ((fixdepth == -1)) ; then
      # Judge last scan based on duration and timeout: Increment success or failure counters
      if ((were_disks_idle_during_scan)) ; then
        log_disk_access_msg="Idle____________"
        ((depth_success_idle_incr_counter++))
        depth_slow_scan_counter=0
      elif (( elapsed_secs < slow_scan_time_limit )) ; then
        # Semi-INVARIANT: ! is_last_depth_scan_timed_out becasue probably slow_scan_time_limit < scan_timeout
        # build cache-pressure by repeating successfull search, and give time for checking for disk-access
        if (( elapsed_time <= avg_elapsed_time + 2*NANO_PR_SEC )) ; then
          # depth_slow_scan_counter because we suspect its a scan which didn't access disks, since scan time was as fast as usual.
          depth_slow_scan_counter=0
          log_disk_access_msg="NonIdleFast_____"
        else
          log_disk_access_msg="NonIdleSlowerAvg"
        fi
        ((depth_success_busy_incr_counter++))
      else
        log_disk_access_msg="NonIdleTooSlow__"
        is_slow_scan=1
        ((depth_slow_scan_counter++))
        depth_success_idle_incr_counter=0
        depth_success_busy_incr_counter=0
        skip_sleep=1
      fi
    fi

    update_cpu_usage
    CPU="$(format_two_digit $cpu_usage)"
    # Note elapsed_secs is an integer, so we need to recompute it from elapsed_time in the awk command below
    avg_text=$(awk "BEGIN{ printf \"%05.2fs, wavg=%05.2fs\n\", ($elapsed_time/$NANO_PR_SEC), ($avg_elapsed_time/$NANO_PR_SEC) ; }")
    log "$start_time_txt Executed $command in (${elapsed_secs}s)${logline_deep_scan_detailed_info} $avg_text ${log_disk_access_msg} $scanned_depth_msg slept ${next_sleep_duration}s Disks idle before/after scan ${time_since_disk_access_before_scan_sec}s/${time_since_disk_access_after_scan_sec}s Scan completed/timedOut counter cnt=$depth_scan_counter/$depth_success_idle_incr_counter/$depth_slow_scan_counter mode=$mode scan_tmo=${scan_timeout_sec}s maxCur=$appliedMaxDepthCurrent maxWeek=$appliedMaxDepth isMaxDepthComputed=$isPreviousMaxDepthComputed CPU=${CPU}%, filecount[$appliedDepth]=${files_at_depth_map[$appliedDepth]}"
    logCsvSimple

    if ((fixdepth == -1)) ; then
      time_since_last_disk_access_sec=$(fnc_time_since_last_disk_access)
      (( time_since_last_disk_access_sec >= min_disk_idle_before_restarting_scan_sec )) && hasBeenIdleLong=1 || hasBeenIdleLong=0
      if (( mode == MODE_IDLE_RESCANNING)) ; then
        no_slow_scans_decrease_depth=$no_slow_scans_decrease_depth_idle
      elif (( mode == MODE_BUSY_RESCANNING)) ; then
        no_slow_scans_decrease_depth=$no_slow_scans_decrease_depth_busy
      else
        no_slow_scans_decrease_depth=$no_slow_scans_decrease_depth_stable
      fi
      ((depth_scan_counter++))
      # appliedMaxDepthCurrent: Used when in mode MODE_BUSY_RESCANNING, and that mode is started when disks where idle during last scan. It has no effect in the other modes I believe.

      # Now increment or decrement depth level based on counters
      if (( mode != MODE_STABLE && (depth_success_idle_incr_counter >= no_idle_scans_increase_depth || depth_success_busy_incr_counter > no_busy_scans_increase_depth) || hasBeenIdleLong && depth_success_idle_incr_counter >= 1 )) ; then
        if (( appliedDepth == maxDepth )) ; then
          isPreviousMaxDepthComputed=1
        fi
        if (( appliedDepth < appliedMaxDepthCurrent || appliedDepth < appliedMaxDepth && (mode == MODE_IDLE_RESCANNING || mode == MODE_STABLE_IDLE) )) ; then
          # Increment depth and check depth_max_incremental_depth: skip straight to maxDepth if deep enough, so we can start sleeping
          appliedDepth=$(depthPlusOne $appliedDepth)
          depth_slow_scan_counter=0
          depth_success_idle_incr_counter=0
          depth_success_busy_incr_counter=0
          depth_scan_counter=0
          if (( hasBeenIdleLong && appliedDepth < appliedMaxDepth )) ; then
            mode=$MODE_IDLE_RESCANNING
          fi
        elif (( (appliedDepth == appliedMaxDepthCurrent && mode == MODE_BUSY_RESCANNING) || appliedDepth == appliedMaxDepth )) ; then
          mode=$MODE_STABLE
        fi
      elif (( (appliedDepth > 0 && depth_slow_scan_counter >= no_slow_scans_decrease_depth || is_last_depth_scan_timed_out) && depth_scan_counter >= min_no_scans_at_new_level )) ; then
        # Sometimes it takes a few scans to build cache pressure which is seen by first scan being very slow, second a bit slow third pretty fast. So never decrease before trying at least 3 scans at level
        depth_slow_scan_counter=0
        depth_success_idle_incr_counter=0
        depth_success_busy_incr_counter=0
        #don't reset 'depth_scan_counter=0' because we are decreasing level
        # Reduce max-depth
        appliedMaxDepthCurrent=$(depthMinusOne "$appliedDepth")
        is_depth_reduced=1
        if (( mode == MODE_STABLE )) ; then
          appliedDepth=$(depthMinus50Percent "$appliedDepth")
        else
          appliedDepth=$(depthMinusOne "$appliedDepth")
          if (( !isPreviousMaxDepthComputed && (mode == MODE_IDLE_RESCANNING || mode == MODE_STABLE_IDLE) )) ; then
            # We were doing a scan during a time when disks have been idle, so system low was probably low. We take this to indicate this is the best depth the system can handle.
            # Set max depth ceiling, which will be rechecked weekly. Move to stable mode, at one level below current level.
            appliedMaxDepth=$appliedMaxDepthCurrent
            isPreviousMaxDepthComputed=1
          fi
          mode=$MODE_STABLE
        fi
      elif (( were_disks_idle_during_scan && (mode == MODE_STABLE || mode == MODE_STABLE_IDLE) && appliedDepth < appliedMaxDepthCurrent )) ; then
        mode=$MODE_BUSY_RESCANNING
      fi

      if (( appliedMaxDepthCurrent < appliedDepth )) ; then
        appliedMaxDepthCurrent="$appliedDepth"
      fi

      # Weekly full scan: In case of reduced max depth, test if we should reset and try going deeper again
      if (( current_time_sec > last_scan_towards_max_depth + frequency_of_full_depth_scan_sec && time_since_last_disk_access_sec >= min_disk_idle_before_restarting_scan_sec )) ; then
        last_scan_towards_max_depth=$current_time_sec
        if (( appliedMaxDepth < maxDepth )) ; then
          log "Starting scheduled depth scan again after waiting ${frequency_of_full_depth_scan_sec}s - disks have been idle for ${time_since_last_disk_access_sec}s"
          appliedMaxDepth=$maxDepth
          isPreviousMaxDepthComputed=0
          scan_timeout_sec=${scan_timeout_sec_idle}
        fi
      fi
      if (( is_depth_reduced )); then
        logLostCacheSimple 1
      elif (( is_slow_scan )); then
        logLostCacheSimple 0
      fi
    fi

    (( (mode == MODE_STABLE || mode == MODE_STABLE_IDLE) && !is_last_depth_scan_timed_out )) && next_sleep_duration=$num_seconds || next_sleep_duration=1
    # we only skip sleep when scan is taking long, so it cannot significantly change impact whether or not we sleep 1 sec. We skip sleep to help cache pressure to build
    if ((!skip_sleep)) ; then
      sleep ${next_sleep_duration}
    fi
    prev_sleep_duration=$next_sleep_duration
  done
  # This line will only be printed if for some odd reason the while loop terminates, without cache_dirs -q being executed (such as user deleting run-file)
  log "cache_dirs stopped"
} &

# while loop was put into background, now disown it
# so it will continue to run when you log off
# to get it to stop, type: rm /var/lock/cache_dirs.LCK

background_pid=$!
echo $background_pid > "${lockfile}"
if [ $background = "no" ] ; then
  # only way to get here is to remove the lock file or kill the background process shell with the while loop
  trap "syslog '$program_name process $background_pid was terminated via trap'; rm -f $lockfile; killtree $background_pid -KILL 2>/dev/null; exit" INT TERM EXIT
  wait
  echo "Stopped"
else
  syslogerror"$program_name process ID $background_pid started"
  disown %%
fi
